# -*- coding: utf-8 -*-
# Lấy lịch chiếu phim từ megastar.vn

import urllib
from urlparse import urljoin 
import cStringIO as StringIO
import traceback
import gzip
import re, os
from lxml import etree
#import lxml.html
import datetime
import hashlib
from pymongo import Connection

dbname = 'Megastar'
conn = Connection('localhost', 27017)
db = conn[dbname]
lich_chieu_phim = db['lichchieuphim']
chi_tiet_phim = db['phim']
#LOCAL_PATH = '/home/namndmk/HarvestedData/Test1/'
LOCAL_PATH = '/home/namndmk/HarvestedData/lichphim/'
baseUrl = 'http://m.megastar.vn/'

def getMD5Hash(textToHash=None):
    return hashlib.md5(textToHash).hexdigest()

def getMd5Path(stringToHash):
    s = getMD5Hash(stringToHash)
    s = '{0}/{1}/{2}/{3}'.format(s[0], s[1], s[2],s[3])
    return s

def getMD5FileName(stringToHash):
    s = getMD5Hash(stringToHash)
    s = '{0}/{1}/{2}/{3}/{4}'.format(s[0], s[1], s[2], s[3],s[4:])
    return s

def saveImage(url):
    try:
        localFilename = '{0}{1}.jpg'.format(LOCAL_PATH, getMD5FileName(url))
        if not os.path.isfile(localFilename):
            if not os.path.exists(LOCAL_PATH + getMd5Path(url)):
                os.makedirs(LOCAL_PATH + getMd5Path(url))
            urllib.urlretrieve(url, localFilename)  
        return localFilename
    except:
        traceback.print_exc()

def regexString(pattern, text):
    try:
        if pattern==None or text==None or pattern=='' or text=='': return
        preg = re.compile(pattern)
        m = preg.search(text)
        if m: return m
    except:
        traceback.print_exc()

def getHTML(url, outputHTML=False):
    try:    
        if type(url).__name__ == 'unicode': url = url.encode('utf-8')
        response = urllib.urlopen(url)
        html = response.read()
        content_type = response.info().getheader('Content-Encoding', '')
        #print 'CONTENT TYPE: ', content_type
        if content_type == 'gzip':
            gzipper = gzip.GzipFile(fileobj=StringIO.StringIO(html))
            html = gzipper.read()
        if outputHTML: print html
        return html 
    except:
        traceback.format_exc()
        
def checkDuplicate(link):
    try:
        if link==None or link=='': return None, None
        reId = regexString('/(\w+)-(\d+)/', link)
        if reId: fId = '{0}-{1}'.format(reId.group(1), reId.group(2)) 
        else: fId = getMD5Hash(link)   
        result= chi_tiet_phim.find_one({'hashUrl' : fId, 'source': 'megastar.vn'})
        if result!=None:  
            print "Dữ liệu đã tồn tại !"
            return 1, fId
        return 0, fId   
    except:
        traceback.print_exc()
        return None, None

def buildTree(html, isXML=False):
    try:
        if isXML==False:
            parser = etree.HTMLParser(encoding='utf-8')
        else:
            parser = etree.XMLParser(encoding='utf-8')
        tree = etree.parse(StringIO.StringIO(html), parser)
        return tree
    except:
        traceback.print_exc()

def getTimesByFilms():
    try:
        cinemaId ={'HCMC - Hùng Vương Plaza':1004,
                   'HCMC - CT Plaza':1006,
                   'HCMC - Parkson Paragon':1007,
                   'Hà Nội - Vincom City Towers':1001,
                   'Hải Phòng - Thùy Dương Plaza':1002,
                   'Đà Nẵng - Vinh Trung Plaza':1005,
                   'Biên Hoà - Siêu thị Co-Op Mart':1003
                   }
        listFilm = getListFilms()
        for k, film in listFilm.items():
            print "Phim: ", k ,film[0]
              
            for key, value in cinemaId.items():
                print 'Rap: ', key
                link = 'msSessionTimeByMovie_Step3.aspx?MovieName={0}&CinemaId={1}&visLang=1'.format(film[0], value)
                link =urljoin(baseUrl, link)
                processLink(link)
                print "----------------"
    except:
        traceback.print_exc()
        
def getListFilms():
    try:
        link = 'http://m.megastar.vn/msSessionTimeByMovie_Step1.aspx?visLang=1'
        html = getHTML(link, outputHTML=False)
        tree = buildTree(html, isXML=False)
        if tree == None: return ''
        contentNode = tree.xpath("//div[@class='content']//div[@id='Movies']")
        listFilms = {}
        tmp = ''
        for item in contentNode[0].xpath(".//li"):
            items = []
            filmNode = item.xpath(".//a")
            film = filmNode[0].text
            tmp = film
            filmLink = filmNode[0].get('href')
            #print filmLink
            if filmLink==None or filmLink=='': return
            preg = re.compile(r'(MovieName=)(.+)(&)')
            m = preg.search(filmLink)
            if m:
                filmName = m.group(2)
                filmName = filmName.replace('&visLang=1','')
                items.append(filmName)
            else:
                print 'Sai regex'
            listFilms[tmp] = items
        return listFilms    
    except:
        traceback.print_exc()
def getListDate():
    try:
        link = 'http://m.megastar.vn/msSessionTimeByDate_Step1.aspx?visLang=1'
        html = getHTML(link, outputHTML=False)
        tree = buildTree(html, isXML=False)
        #print etree.tostring(tree)
        if tree == None: return ''
        contentNode = tree.xpath("//div[@class='content']//div[@id='ListDates']")
        dates = {}
        tmp = ''
        for item in contentNode[0].xpath(".//li"):
            items = []
            dateNode = item.xpath(".//a")
            date = dateNode[0].text
            tmp = date
            dateLink = dateNode[0].get('href')
            dateLink =dateLink.replace("Step2","Step3")
            #print dateLink
            items.append(dateLink)
            dates[tmp] = items
            #items = urllib.urlencode(items.append(date))         
        return dates
    except:
        traceback.print_exc()
      
def getTimesByDate():
    try:
        print "LICH CHIEU PHIM THEO NGAY: "
        #today = datetime.datetime.now().strftime("%Y-%m-%d")
        cinemaId ={'HCMC - Hùng Vương Plaza':1004,
                   'HCMC - CT Plaza':1006,
                   'HCMC - Parkson Paragon':1007,
                   'Hà Nội - Vincom City Towers':1001,
                   'Hải Phòng - Thùy Dương Plaza':1002,
                   'Đà Nẵng - Vinh Trung Plaza':1005,
                   'Biên Hoà - Siêu thị Co-Op Mart':1003
                   }
        for key, value in cinemaId.items():
            print 'Rap: ', key
            dateL = getListDate()
            for k, v in dateL.items():
                print k
                dlink = '/{0}&CinemaId={1}'.format(v[0], value)
                processLink(dlink)
                print '..................'   
    except:
        traceback.print_exc()

def getTimesByCinema():
    try:
        print "LICH CHIEU PHIM THEO RAP: "
        cinemaId ={'HCMC - Hùng Vương Plaza':1004,
                   'HCMC - CT Plaza':1006,
                   'HCMC - Parkson Paragon':1007,
                   'Hà Nội - Vincom City Towers':1001,
                   'Hải Phòng - Thùy Dương Plaza':1002,
                   'Đà Nẵng - Vinh Trung Plaza':1005,
                   'Biên Hoà - Siêu thị Co-Op Mart':1003
                   }
        for key, value in cinemaId.items():
            print 'Rap: ', key
            link = 'msSessionTimeByCinema_Step2.aspx?CinemaId={0}&visLang=1'.format(value)
            link = urljoin(baseUrl, link)
            html = getHTML(link)
            tree = buildTree(html, isXML=False)
            if tree == None: print "Khong dung duoc cay"; return;
            contentNode = tree.xpath("//div[@class='content']//div[@id='Movies']/*")
            for item in contentNode[0].xpath("//li/a"):
                print 'ten Phim: ', item.text
                linkF = item.get('href')
                processLink(linkF)
                print '..................'         
            print '******************************'
        print "----------------------------------------------------------------- "
        print "---------------------------------------------------------------------------- "
    except:
        traceback.print_exc() 
        
def processLink(link):
    try:
        link = urljoin(baseUrl, link)
        html = getHTML(link)
        tree = buildTree(html, isXML=False)
        if tree == None: print "Not build Tree"; return;
        contentNode = tree.xpath("//div[@class='content']//div[@id='SesstionTime']/*")
        if len(contentNode)>0:
            dictList = {}
            tmp = "";
            for i in range(0, len(contentNode)):
                node = contentNode[i];
                if node.get("class") == "tab":
                    tmp = node.text 
                elif node.get("class") == "showtime_c":
                    listNode = node.xpath("./div/a")
                    items = []
                    for item in listNode:
                        ctext = item.text
                        items.append(ctext)
                    dictList[tmp] = items
            #print dict          
            for key, value in dictList.items():
                '''if key == datetime :
                    print 'Ngay: ', key
                    print 'Gio chieu: '
                    for i in range(0, len(value)):
                        print  value[i] 
                else:'''
                print 'Ngay: ', key
                print 'Gio chieu: '
                for i in range(0, len(value)):
                    print  value[i]
        else:
            print "Chua cap nhat thong tin"
            
    except:
        traceback.print_exc()
        
def processXMLFilm(film):
    try:
        cinemaId ={
                   'HCMC - Hùng Vương Plaza':1004,
                   'HCMC - CT Plaza':1006,
                   'HCMC - Parkson Paragon':1007,
                   'Hà Nội - Vincom City Towers':1001,
                   'Hải Phòng - Thùy Dương Plaza':1002,
                   'Đà Nẵng - Vinh Trung Plaza':1005,
                   'Biên Hoà - Siêu thị Co-Op Mart':1003
                   }
        for key, value in cinemaId.items():
            print "Rap: ", key
            urlXML = 'http://www.megastar.vn/megastarXMLData.aspx?RequestType=GetSessionTimes&CinemaID={0}&MovieName={1}&Time=TodayAndTomorrow&visLang=1'.format(value, film)
            #print urlXML
            html = getHTML(urlXML, outputHTML=False)
            tree = buildTree(html, isXML=True)
            #print etree.tostring(tree)
            if tree == None: print 'Khong dung duoc cay'
            contentNode = tree.xpath("//movies/date")
            #print len(contentNode)
            for item in contentNode:
                dateFilm = item.get('name')
                print "Ngay: ", dateFilm
                timeNode = item.xpath(".//time/date")
                if len(timeNode) > 0:
                    for iTime in timeNode:
                        sTime = iTime.text
                        sTime = sTime[11:].replace(":00","")
                        print sTime    
                else: print "sai xpath"
    except:
        traceback.print_exc()
          
def getNowShowning():
    try:
        print "PHIM DANG CHIEU :"
        link = 'http://www.megastar.vn/vn/nowshowing/'
        html = getHTML(link)
        tree = buildTree(html, isXML=False)
        if tree == None: return ''
        contentNode = tree.xpath("//div[@id='body']//div[@class='landingbody']")
        if len(contentNode)>0:
            for itemNode in contentNode[0].xpath(".//div[@class='landingbody_item']"): 
                imageNode = itemNode.xpath(".//a/img")
                #print "image link ",  imageNode[0].get('src')
                imageLink =   urljoin('http://megastar.vn',imageNode[0].get('src'))
                pathImage = saveImage(imageLink) 
                #print "image link: ", pathImage            
                titleNode = itemNode.xpath(".//div[@class='m_center']/a/span")
                if len(titleNode)>0:
                    title = titleNode[0].text
                    #print title
                desNode = itemNode.xpath(".//div[@class='m_center']/a/following-sibling::*")
                ctext = []
                for itemDes in desNode:
                    strd = ""
                    if itemDes.text != None and len(itemDes.text) > 0:
                        strd = re.sub(r"\s+", " ", itemDes.text)
                        ctext.append(strd)
                    elif itemDes.tail != None and len(itemDes.tail) > 0:
                        strd = re.sub(r"\s+", " ", itemDes.tail)
                        ctext.append(strd)
                '''for item in ctext:
                    #print item
                    m1 = regexString(unicode("Đạo diễn:(.+)"), item)
                    if m1:
                        print "Đạo diễn: ", m1.group(1)
                    m2 = regexString(unicode("Thời lượng:(.+)"), item)
                    if m2:
                        print "Thời lượng: ",m2.group(1)
                    m3 = regexString(unicode("Thể loại:(.+)"), item)
                    if m3:
                        print "Thể loại: ",m3.group(1)
                print "Mô tả: ", ctext[3]
                print ctext[4]  ''' 
                linkNode = itemNode.xpath('.//div[@class="q_session_time"]/a[1]')
                dLink = linkNode[0].get('href')
                check_exists, fId    = checkDuplicate(dLink)
                if check_exists==1: return 1
                #fId = getMD5Hash(dLink)
                #print fId
                exNode = itemNode.xpath('.//div[@class="q_session_time"]/a[2]')
                if len(exNode) > 0:
                    film = processTitleFilm(exNode[0].get("onclick"))
                    #print film
                    #print "Xem lich chieu: "
                    processXMLFilm(film)
                doc= ({ 'hashUrl': fId,
                         'title' : title,
                         'thumbnailImage' : pathImage,
                         'description' : ctext,
                         'source' : 'megastar.vn',
                    })
                chi_tiet_phim.save(doc)
                print "-----------------------------------"          
    except:
        traceback.print_exc()
        
def processTitleFilm(text):
    try:
        #print type(text).__name__, text
        if text==None or text=='': return
        preg = re.compile(r'session\("(.+)",')
        m = preg.search(text)
        if m:
            return m.group(1)
        else:
            print 'Sai regex'
    except:
        traceback.print_exc()
        

def getLink():
    try:
        link = 'http://www.megastar.vn/vn/nowshowing/'
        html = getHTML(link)
        tree = buildTree(html)
        if tree == None: return ''
        contentNode = tree.xpath("//div[@id='body']//div[@class='landingbody']")
        data = {}
        temp = ''
        if len(contentNode) > 0 :
            '''
            aImage = []
            for imgNode in contentNode[0].xpath(".//div[@class='landingbody_item']"):
                imageNode = imgNode.xpath(".//a/img")
                #print "image link ",  imageNode[0].get('src')
                imageLink =   urljoin('http://megastar.vn',imageNode[0].get('src'))
                #pathImage = saveImage(imageLink)
                aImage.append(imageLink)
                ''' 
            for itemNode in contentNode[0].xpath(".//div[@class='landingbody_item']//div[@class='m_right']"):
                titleNode = itemNode.xpath(".//div[@class='q_session_time']/a[1]")
                temp = titleNode[0].get('href')
                detailNode = itemNode.xpath(".//div[@class='q_session_time']/a[1]")
                items = [] 
                if len(detailNode) > 0:
                    detailLink = detailNode[0].get('href')
                    items.append(detailLink)
                    data[temp] =items
        return data   
    except:
        traceback.print_exc()
        
def getLichChieu():
    try:
        print ''
        
    except:
        traceback.print_exc()
        
def getDetailFilms():
    try:
        baseLink = 'http://www.megastar.vn'
        dataLink = getLink()
        #print dataLink
        for k, value in dataLink.items():
            film =value[0]
            film = film[4:]
            imageLink = '/Images/Movies/LargeNew/{0}.jpg'.format(film)
            imageLink =   urljoin('http://megastar.vn',imageLink)
            #print imageLink
            data =saveImage(imageLink)
            detailImage = {'data':data,
                           'type':'image',
                           'caption':''
                           }
            detailLink = urljoin(baseLink,value[0])
            html = getHTML(detailLink)
            check_exists, fId    = checkDuplicate(detailLink)
            if check_exists==1: return 1
            tree = buildTree(html)
            if tree == None: return ''
            contentNode = tree.xpath('//div[@class="left_col"]/div[@class="content_md"]')
            #print len(contentNode)
            if len(contentNode)>0:
                titleNode = contentNode[0].xpath(".//h2/span")
                if len(titleNode)>0:
                    titleFilm = titleNode[0].text
                    print titleFilm
                diNode = contentNode[0].xpath(".//div[1]/span[2]")
                duration = diNode[0].text
                if duration == None or duration == '':
                    duration = 'Chua cap nhat'
                print duration
                typeNode = contentNode[0].xpath(".//div[2]//span[@id='ContentPlaceHolder1_lblGenre']")
                #print len(typeNode)
                category = typeNode[0].text
                print category
                desNode = contentNode[0].xpath("./span[1]")
                description = desNode[0].text
                print description
                dNode = contentNode[0].xpath(".//span[@id='ContentPlaceHolder1_lblDirector']")
                director = dNode[0].text
                print director
                aNode = contentNode[0].xpath(".//span[@id='ContentPlaceHolder1_lblCast']")
                actor = aNode[0].text
                print actor
                lNode = contentNode[0].xpath(".//span[@id='ContentPlaceHolder1_lblLanguage']")
                language = lNode[0].text
                print language
                dNode = contentNode[0].xpath(".//span[@id='ContentPlaceHolder1_lblReleaseDate']")
                sDate = dNode[0].text
                if sDate == None or sDate == '':
                    sDate = 'Chua cap nhat'
                print sDate
            desdoc= ({ "hashUrl": fId,
                       "title" : titleFilm,
                       "detail" : detailImage,
                       "description" : description,
                       "category":category,
                       "duaration":duration,
                       "country":"",
                       "actor":actor,
                       "director":director,
                       "source" : "megastar.vn",
                    })
            chi_tiet_phim.save(desdoc)
    except:
        traceback.print_exc()

if __name__ == '__main__':
    #print getListFilms()
    #getListDate()
    #getTimesByCinema()
    #getTimesByDate()
    #getTimesByFilms()
    #getNowShowning()
    #processXMLFilm('The+Twilight+Series:+Breaking+Dawn+Part+1')
    #getLink()
    getDetailFilms()
    os._exit(1)

